from transformers import AutoTokenizer

model_path = r"../../../pretrained_model/IDEA-CCNL/Erlangshen-Roberta-110M-Sentiment"
# model_path = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_path)
sequence = "今天心情真好。"

if __name__ == "__main__":
    print(tokenizer(sequence))
    print(tokenizer.encode(sequence))
    print(tokenizer.encode_plus(sequence))
